# -*- coding: utf-8 -*-
"""
Created on Wed Aug  3 14:34:11 2016

@author: XT-YY

get fund data from sohu and ifeng
"""
import urllib, json
from splinter import Browser
import pandas as pd
import time

dst_path = ""

if __name__ == "__main__":
    sdate = "20150101"
    edate = "20160802"
    col_name = ["Date", "Open", "Close", "Change", "PCT_Change", "Low", "High", 
                "Volume", "Amount", "TurnOver"]
    
    # get the market data
    codeList = ['511880', '511990', '204001']
    for code in codeList:
        url = ("http://q.stock.sohu.com/hisHq?code=cn_" + code + "&start=" 
              + sdate + "&end=" + edate + "&stat=1&order=D&period=d")
        response = urllib.request.urlopen(url);
        time.sleep(1)
        text = response.read().decode("gbk")
        data = json.loads(text, encoding="gbk")
        data = data[0]['hq']
        df = pd.DataFrame(data, columns=col_name)
        df["#Code"] = code
        df = df[["#Code", "Date", "Open", "High", "Low", "Close", "Change", 
                "PCT_Change", "Volume", "Amount"]]
        df.sort_values(by="Date", ascending=True, inplace=True)
        if code == "511880":
            df_part = df.copy()
            continue
        df.to_csv(dst_path + code + ".csv", index=False)
    
    # get the net value data
    sdate = "20141231"  # 凤凰财经的bug 
    code = "511880"
    url = ("http://app.finance.ifeng.com/data/fund/jjjz.php?symbol=" + code +
            "&begin_day=" + sdate + "&end_day=" + edate)
    browser = Browser('phantomjs', user_agent="Mozilla/5.0 (Windows NT 6.1; WOW64) \
    AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36")
    browser.driver.maximize_window()
    browser.visit(url)
    time.sleep(3)
    xpath = '/html/body/div[3]/div/div[3]/div[3]/div[2]/table/tbody'
    table = browser.find_by_xpath(xpath).first.text.replace("  ", " ").split('\n')
    values = [x.split(' ') for x in table[1:]]
    col_name = ["Date", "AnnounDate", "NetValue", "Change", "PCT_Change", "AccumulatedNetValue"]
    df = pd.DataFrame(values, columns=col_name)
    df = df[["Date", "NetValue", "AccumulatedNetValue"]]
    df.sort_values(by="Date", ascending=True, inplace=True)
    df.set_index("Date", inplace=True)
    df_part.set_index("Date", inplace=True)
    dfs = pd.concat([df_part, df], axis=1)
    dfs.index.name = "Date"
    dfs.reset_index(inplace=True)
    dfs = dfs[["#Code", "Date", "Open", "High", "Low", "Close", "Change", 
             "PCT_Change", "Volume", "Amount", "NetValue", "AccumulatedNetValue"]]
    dfs.to_csv(dst_path + code + ".csv", index=False)
    
    
    
    
    
    
    
